#ifndef RED_SYSTEM_TOKENIZER_H
#define RED_SYSTEM_TOKENIZER_H

#include <cctype>
#include "LLVM.h"
#include "Token.h"
#include "Logger.h"

class Compilation;

class Tokenizer
{
private:
    Compilation &comp;
    Logger LOG;

public:
    Tokenizer(Compilation &c) : comp(c) {};
    Tokenizer(Compilation &&) = delete;  // prevents rvalue binding

    Token * tokenize(const char *buffer, Token **end_token);

private:
    Token * newToken(TokenBuffer &buf, TokenId id, const char *start, int len) {
        Token *t = buf.alloc();
        t->id = id;
        t->len = len;
        t->loc = start;
        return t;
    }

    template <typename... Args>
    Token * tokenError(const char *loc, unsigned diag_id, Args &&... args);
};

#endif /* RED_SYSTEM_TOKENIZER_H */
